from lxml import etree
import DBUtil
import Sha1Util
def processingData(response):
    '''
    获取controller的数据，处理数据，数据都交由db,写入数据库
    :param respomse:
    :return:
    '''
    page_text = response.text
    tree = etree.HTML(page_text)
    city_name_list = tree.xpath('/html/body/section/div/ul/li/ul/li/a/text()')  #获取城市名称
    city_href_list = tree.xpath('//ul[@class="city_table"]//a/@href') #获取城市连接
    city_dict = dict(zip(city_name_list,city_href_list)) #将两个列表合并成一个字典
    #创建数据库
    sql="""
        CREATE TABLE if not exists `city`(
        `id` varchar(255) NOT NULL,
        `name` varchar(20) NOT NULL,
        `url` varchar(50) DEFAULT NULL,
        `status` int(16) NOT NULL DEFAULT 0 comment '0表示为获取区县数据，1为已获取，-1位获取失败',
          PRIMARY KEY (`id`)
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
     """
    DBUtil.createDatabase(sql)

    for k,v in city_dict.items():
        id = Sha1Util.jia_mi(k+v)
        print(id)
        sql = 'insert ignore into city (id,name,url) values ("%s","%s","%s")' % (id,k,v)
        DBUtil.modifyData(sql)